{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e4d4f550-fbad-45e7-b852-542c2f3d9ed9",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "通过指定tree来解析：\n",
      "Big data news\n",
      "AI news\n",
      "2018.8.1\n"
     ]
    }
   ],
   "source": [
    "import html5lib\n",
    "print('通过指定tree来解析：')\n",
    "document='<html><head><title>Test</title></head><body><h1 align=\"center\">Big data news</h1><h1 align=\"center\">AI news</h1><h1 align=\"right\">2018.8.1</h1></body></html>'\n",
    "#构造HTMLParser实例，指定构造lxml的树\n",
    "p=html5lib.HTMLParser(strict=False,tree=html5lib.getTreeBuilder('lxml'),namespaceHTMLElements=False)\n",
    "#解析HTML文档\n",
    "t=p.parse(document)\n",
    "rows=t.xpath('/html/body/h1')\n",
    "for row in rows:\n",
    "  t=row.xpath('./text()')[0]\n",
    "  print(t)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "02ac53eb-8b70-4b70-a5ab-a32391179214",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "通过指定treebuilder来解析：\n",
      "Big data news\n",
      "AI news\n",
      "2018.8.1\n"
     ]
    }
   ],
   "source": [
    "import html5lib\n",
    "print('通过指定treebuilder来解析：')\n",
    "document='<html><head><title>Test</title></head><body><h1 align=\"center\">Big data news</h1><h1 align=\"center\">AI news</h1><h1 align=\"right\">2018.8.1</h1></body></html>'\n",
    "#直接调用html5lib.parse来解析，解析时采用lxml构建树的方法\n",
    "content=html5lib.parse(document,treebuilder=\"lxml\",namespaceHTMLElements=False)\n",
    "#指定要提取的内容所在的标签路径\n",
    "rows=content.xpath('/html/body/h1')\n",
    "for row in rows:\n",
    "  t=row.xpath('./text()')[0]    #定位到标签节点后，通过text()提取内容\n",
    "  print(t)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "08375640-a8a2-4512-94f9-323554c5cf53",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "处理标签不完整或有错误的HTML：\n",
      "Big data news\n",
      "AI news\n",
      "2018.8.1\n"
     ]
    }
   ],
   "source": [
    "import html5lib\n",
    "print('处理标签不完整或有错误的HTML：')\n",
    "#这个HTML文档中有不完整的标签：缺少一个</h1>、有错误的标签：</m1>\n",
    "document='<html><head><title>Test</title></head><body><h1 align=\"center\">Big data news</h1><h1 align=\"center\">AI news<h1 align=\"right\">2018.8.1</m1></body></html>'\n",
    "p=html5lib.HTMLParser(strict=False,tree=html5lib.getTreeBuilder('lxml'),namespaceHTMLElements=False)\n",
    "t=p.parse(document)\n",
    "rows=t.xpath('/html/body/h1')\n",
    "for row in rows:\n",
    "  t=row.xpath('./text()')[0]\n",
    "  print(t)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "ab2f5c42-3d14-432e-a13b-1332681cf5c1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<a class=\"py1\" href=\"http://www.icourse163.org/course/BIT-268001\" id=\"link1\">Basic Python</a>,\n",
       " <a class=\"py2\" href=\"http://www.icourse163.org/course/BIT-1001870001\" id=\"link2\">Advanced Python</a>]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import requests\n",
    "r = requests.get(\"http://python123.io/ws/demo.html\")\n",
    "r.text\n",
    "demo = r.text\n",
    "from bs4 import BeautifulSoup\n",
    "soup = BeautifulSoup(demo,\"html.parser\")\n",
    "soup.find_all('a')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "fdfee166-93e0-4fb9-88e7-ad96b8aafc07",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<b>The demo python introduces several python courses.</b>,\n",
       " <a class=\"py1\" href=\"http://www.icourse163.org/course/BIT-268001\" id=\"link1\">Basic Python</a>,\n",
       " <a class=\"py2\" href=\"http://www.icourse163.org/course/BIT-1001870001\" id=\"link2\">Advanced Python</a>]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.find_all(['a','b'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "3a9a8df6-9136-4390-96a9-23e750057a80",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "html\n",
      "head\n",
      "title\n",
      "body\n",
      "p\n",
      "b\n",
      "p\n",
      "a\n",
      "a\n"
     ]
    }
   ],
   "source": [
    "for tag in soup.find_all(True):\n",
    "    print(tag.name)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cf3db879-7b39-4675-9c82-a0c0656fa702",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "body\n",
      "b\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "for tag in soup.find_all(re.compile('b')):\n",
    "      print(tag.name)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "03089b67-2dd0-4726-8db0-4ceb676ae7f8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<p class=\"course\">Python is a wonderful general-purpose programming language. You can learn Python from novice to professional by tracking the following courses:\n",
       " <a class=\"py1\" href=\"http://www.icourse163.org/course/BIT-268001\" id=\"link1\">Basic Python</a> and <a class=\"py2\" href=\"http://www.icourse163.org/course/BIT-1001870001\" id=\"link2\">Advanced Python</a>.</p>]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.find_all('p','course')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "69795093-caf5-4b75-922c-456f723f4a26",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<a class=\"py1\" href=\"http://www.icourse163.org/course/BIT-268001\" id=\"link1\">Basic Python</a>]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.find_all(id='link1')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "5a4796c6-4d46-4cfc-93c2-efe4e90f19c9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.find_all(id='link')   #必须完整准确搜索\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "f70381a5-7c99-4e2f-8c7d-7e9f065b0677",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<a class=\"py1\" href=\"http://www.icourse163.org/course/BIT-268001\" id=\"link1\">Basic Python</a>,\n",
       " <a class=\"py2\" href=\"http://www.icourse163.org/course/BIT-1001870001\" id=\"link2\">Advanced Python</a>]"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re   #直接给出一部分\n",
    "soup.find_all(id=re.compile('link'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "8d8b03fd-1fef-4360-bc13-e1fe7867d7f1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.find_all('a')\n",
    "soup.find_all('a',recursive=False)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "dd1e58fc-2583-4ed5-b331-594435f08db7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Basic Python']"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.find_all(string=\"Basic Python\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "eb48c37d-351f-43d5-a333-316a5eccf960",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Python is a wonderful general-purpose programming language. You can learn Python from novice to professional by tracking the following courses:\\r\\n',\n",
       " 'Basic Python',\n",
       " 'Advanced Python']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import re\n",
    "soup.find_all(string = re.compile(\"Python\"))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "52503c88-526b-4567-9d1f-415ec56d6e06",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
